NUMA node migration

Adds NUMA node migration based on live migration to
xend. By adding another parameter to "xm migrate" the target NUMA node
number gets propagated to the target host (can be both localhost or a
remote host). The restore function then sets the VCPU affinity
accordingly. Only changes Python code in xend. I hope that the patch
doesn't break XenAPI compatibility (adding a parameter seems fine?).

# xm migrate --live --node=<nodenr> <domid> localhost
<nodenr> is the number as shown with 'xm info' under node_to_cpu

I am aware that using live migration isn't the best approach (takes
twice the memory and quite some time), but it's less intrusive and
works fine (given localhost migration stability...)

Signed-off-by: Andre Przywara <andre.przywara@amd.com>

diff --git a/tools/python/xen/xend/XendAPI.py b/tools/python/xen/xend/XendAPI.py
index c31f3657f2780a11f9326dc761140b422e9ff2c7..354e9aab8ab2ed3b8f72e06655e0a9b0de86303c 100644 (file)
--- a/tools/python/xen/xend/XendAPI.py
+++ b/tools/python/xen/xend/XendAPI.py
@@ -1761,9 +1761,10 @@ class XendAPI(object):
 
         resource = other_config.get("resource", 0)
         port = other_config.get("port", 0)
+        node = other_config.get("node", 0)
         
         xendom.domain_migrate(xeninfo.getDomid(), destination_url,
-                              bool(live), resource, port)
+                              bool(live), resource, port, node)
         return xen_api_success_void()
 
     def VM_save(self, _, vm_ref, dest, checkpoint):

diff --git a/tools/python/xen/xend/XendCheckpoint.py b/tools/python/xen/xend/XendCheckpoint.py
index adc63c0e6358ca0030565b26b950845158174735..a27c8d934d79eb55fb49cccc0ff5d79541ff70fb 100644 (file)
--- a/tools/python/xen/xend/XendCheckpoint.py
+++ b/tools/python/xen/xend/XendCheckpoint.py
@@ -22,6 +22,7 @@ from xen.xend.XendError import XendError, VmError
 from xen.xend.XendLogging import log
 from xen.xend.XendConfig import XendConfig
 from xen.xend.XendConstants import *
+from xen.xend import XendNode
 
 SIGNATURE = "LinuxGuestRecord"
 QEMU_SIGNATURE = "QemuDeviceModelRecord"
@@ -56,10 +57,23 @@ def read_exact(fd, size, errmsg):
     return buf
 
 
-def save(fd, dominfo, network, live, dst, checkpoint=False):
+def insert_after(list, pred, value):
+    for i,k in enumerate(list):
+        if type(k) == type([]):
+           if k[0] == pred:
+              list.insert (i+1, value)
+    return
+
+
+def save(fd, dominfo, network, live, dst, checkpoint=False, node=-1):
     write_exact(fd, SIGNATURE, "could not write guest state file: signature")
 
-    config = sxp.to_string(dominfo.sxpr())
+    sxprep = dominfo.sxpr()
+
+    if node > -1:
+        insert_after(sxprep,'vcpus',['node', str(node)])
+
+    config = sxp.to_string(sxprep)
 
     domain_name = dominfo.getName()
     # Rename the domain temporarily, so that we don't get a name clash if this
@@ -192,6 +206,21 @@ def restore(xd, fd, dominfo = None, paused = False, relocating = False):
     else:
         dominfo = xd.restore_(vmconfig)
 
+    # repin domain vcpus if a target node number was specified 
+    # this is done prior to memory allocation to aide in memory
+    # distribution for NUMA systems.
+    nodenr = -1
+    for i,l in enumerate(vmconfig):
+        if type(l) == type([]):
+            if l[0] == 'node':
+                nodenr = int(l[1])
+
+    if nodenr >= 0:
+        node_to_cpu = XendNode.instance().xc.physinfo()['node_to_cpu']
+        if nodenr < len(node_to_cpu):
+            for v in range(0, dominfo.info['VCPUs_max']):
+                 xc.vcpu_setaffinity(dominfo.domid, v, node_to_cpu[nodenr])
+
     store_port   = dominfo.getStorePort()
     console_port = dominfo.getConsolePort()
 

diff --git a/tools/python/xen/xend/XendDomain.py b/tools/python/xen/xend/XendDomain.py
index 2f1b1fef4c4efbaae4101be8a027ce0acb01aaa5..0fcdbc3c1f18276eeb15d2d8488b232c60eb18f1 100644 (file)
--- a/tools/python/xen/xend/XendDomain.py
+++ b/tools/python/xen/xend/XendDomain.py
@@ -1258,7 +1258,7 @@ class XendDomain:
 
         return val       
 
-    def domain_migrate(self, domid, dst, live=False, resource=0, port=0):
+    def domain_migrate(self, domid, dst, live=False, resource=0, port=0, node=-1):
         """Start domain migration.
         
         @param domid: Domain ID or Name
@@ -1271,6 +1271,8 @@ class XendDomain:
         @type live: bool
         @keyword resource: not used??
         @rtype: None
+        @keyword node: use node number for target
+        @rtype: int 
         @raise XendError: Failed to migrate
         @raise XendInvalidDomain: Domain is not valid        
         """
@@ -1299,7 +1301,7 @@ class XendDomain:
 
         sock.send("receive\n")
         sock.recv(80)
-        XendCheckpoint.save(sock.fileno(), dominfo, True, live, dst)
+        XendCheckpoint.save(sock.fileno(), dominfo, True, live, dst, node=node)
         sock.close()
 
     def domain_save(self, domid, dst, checkpoint=False):

diff --git a/tools/python/xen/xm/migrate.py b/tools/python/xen/xm/migrate.py
index 5a94f6c4d58182d444189f78ddfbfb2799aa0ee6..535cc3a2d0ccbab13f0e2368b472c64171f6f7bf 100644 (file)
--- a/tools/python/xen/xm/migrate.py
+++ b/tools/python/xen/xm/migrate.py
@@ -43,6 +43,10 @@ gopts.opt('port', short='p', val='portnum',
           fn=set_int, default=0,
           use="Use specified port for migration.")
 
+gopts.opt('node', short='n', val='nodenum',
+          fn=set_int, default=-1,
+          use="Use specified NUMA node on target.")
+
 gopts.opt('resource', short='r', val='MBIT',
           fn=set_int, default=0,
           use="Set level of resource usage for migration.")
@@ -65,11 +69,13 @@ def main(argv):
         vm_ref = get_single_vm(dom)
         other_config = {
             "port":     opts.vals.port,
-            "resource": opts.vals.resource
+            "resource": opts.vals.resource,
+            "node":     opts.vals.node
             }
         server.xenapi.VM.migrate(vm_ref, dst, bool(opts.vals.live),
                                  other_config)
     else:
         server.xend.domain.migrate(dom, dst, opts.vals.live,
                                    opts.vals.resource,
-                                   opts.vals.port)
+                                   opts.vals.port,
+                                   opts.vals.node)